/*
 * @(#)HypertextUnit.java	1.01 11/27/07
 *
 * Copyright 2007 Fabio Gasparetti. All rights reserved.
 */

package org.tabularium.text.html;

import java.util.*;

import java.util.logging.LogManager;
import java.io.*;
import java.net.URL;
import java.net.MalformedURLException;

import org.tabularium.net.downloader.Downloader;
import org.tabularium.text.analysis.*;

/**
 * A node in a tree that represents part of Html page's content.
 * <p>
 * Actually it corresponds with a non-abstract class, but we keep it abstract in
 * order to allow sub-classing.
 * <p>
 * The node should correspond to a visual html block, e.g., table, column, text.
 * Each node has the following features:
 * <ul>
 * <li><em>id</em>: an univoke identification string
 * <li><em>url</em>: an absolute-url (i.e. with the hostname) if the node
 * correspond to a link
 * <li><em>text</em>: the text of the node
 * <li><em>type</em>: the type of the node
 * </ul>
 * The node type is a string corresponding to one of the html delimitators
 * defined in <code>HypertextUnitBuilder</code>, or to one of the following
 * types: <code>ROOT, TEXT, IMG, LINK</code>.
 * <p>
 * The text is not null iff the type of the unit is:
 * <ul>
 * <li><code>TEXT</code>
 * <li><code>TITLE</code>
 * <li><code>IMG</code> and the <code>ALT</code> attribute is not empty.
 * <li><code>LINK</code> and the <code>TITLE</code> attribute is not empty.
 * </ul>
 * <p>
 * In other cases, such as LINK or TR, the text is to be found in the immediate
 * descendants of these units.
 * <p>
 * Each node has a list of descendants and a reference to the parent in the
 * tree.
 * <p>
 * The id of a node follows the pattern:
 * <p>
 * <em>id1:id2:id3 ...</em>
 * <p>
 * where <em>id3</em> is the index (an integer starting from 0) of the node in
 * the 3rd level, in the descendant list of the node <em>id2</em> in the 2nd
 * level.
 * <p>
 * See <code>HypertextUnitBuilder</code> implementations for details on the tree
 * building of a Web page.
 * <p>
 * Remember to extend the clone methods in derived classes if you do not want to
 * share Collection objects.
 * 
 * @author Fabio Gasparetti
 * @version 1.0, 05/01/07
 */
public abstract class HypertextUnit implements Serializable, Cloneable,
		Comparable {
	static final long serialVersionUID = -4343639226279046243L;
	
	public final static int DEPTH_FIRST_TRAVERSAL = 1;

	public final static int BREADTH_FIRST_TRAVERSAL = 2;

	/**
	 * Alternative text from links (i.e., title elements), images, AREA, INPUT
	 * and APPLET are ignored
	 */
	public final static int ALTTXT_IGNORE = 1;

	/**
	 * Alternative text from links (i.e., title elements), images, AREA, INPUT
	 * and APPLET are always included
	 */
	public final static int ALTTXT_INLCUDE = 2;

	/**
	 * Alternative text from links are included if anchor is empty while text
	 * ALT text from images, AREA, INPUT and APPLET are included
	 */
	public final static int ALTTXT_INCLUDEONEMPTY = 3;

	/** Descendants as a list of HypertextUnit objects. */
	protected List content = new ArrayList();

	/** The parent of the node, null is the unit is root */
	transient protected HypertextUnit parent = null;

	/** Id of the unit */
	protected String id = null;

	/** url of the unit if it is a link or image */
	protected String url = null;

	/** text of the unit */
	protected String text = null;

	/**
	 * The name of the tag that generates the unit, or one of the following
	 * strings: <code>ROOT, TEXT, IMG, LINK</code>
	 */
	protected String type = null;

	/**
	 * The potential attributes in the given tag (e.g, style, class). TITLE and
	 * ALT attributes are stored in a different location and they can be
	 * retrieved by calling {@link #getText()}.
	 */
	transient protected HypertextUnitAttribute[] attributes = new HypertextUnitAttribute[0];

	/** Absolute positioning. */
	protected int offsetLeft = -1;

	/** Absolute positioning. */
	protected int offsetTop = -1;

	/** Width of the unit. */
	protected int width = -1;

	/** Height of the unit. */
	protected int height = -1;

	/**
	 * Returns the unit url if exists or null otherwise.
	 */
	public String getUrl() {
		return url;
	}

	/**
	 * Returns the unit id.
	 */
	public String getId() {
		return id;
	}

	/**
	 * If the HypertextUnitBuilder is able to render the unit, the method
	 * returns the width of the graphical element.
	 * 
	 * Note: <code>HypertextUnitBuilder</code> instances may ignore this field;
	 * in this case <code>-1</code> is the default value.
	 */
	public int getWidth() {
		return width;
	}

	/**
	 * If the HypertextUnitBuilder is able to render the unit, the method
	 * returns the height of the graphical element.
	 * 
	 * Note: <code>HypertextUnitBuilder</code> instances may ignore this field;
	 * in this case <code>-1</code> is the default value.
	 */
	public int getHeight() {
		return height;
	}

	/**
	 * If the HypertextUnitBuilder is able to render the unit, the method
	 * returns the absolute positioning of the graphical element.
	 * 
	 * Note: <code>HypertextUnitBuilder</code> instances may ignore this field;
	 * in this case <code>-1</code> is the default value.
	 */
	public int getOffsetTop() {
		return offsetTop;
	}

	/**
	 * If the HypertextUnitBuilder is able to render the unit, the method
	 * returns the absolute positioning of the graphical element.
	 * 
	 * Note: <code>HypertextUnitBuilder</code> instances may ignore this field;
	 * in this case <code>-1</code> is the default value.
	 */
	public int getOffsetLeft() {
		return offsetLeft;
	}

	/**
	 * Returns the text of the unit.
	 * <p>
	 * Beware of some tags, such as LINK or TR. The text is to be found in the
	 * immediate descendants of these units. Use <code>enclosedText()</code>.
	 * 
	 * @see HypertextUnit#enclosedText().
	 */
	public String getText() {
		return text;
	}

	/**
	 * Returns the text of the unit considering also the direct descendants,
	 * i.e., one level in depth. Useful for tags such as LINK and TR.
	 * 
	 * @see HypertextUnit#getText().
	 * @deprecated As for version 1.1, replaced by {@link #getEnclosedText(int)}
	 *             .
	 */
	public String getEnclosedText() {
		return getEnclosedText(HypertextUnit.ALTTXT_INLCUDE);
	}

	/**
	 * Returns the text of the unit considering also the direct descendants,
	 * i.e., one level in depth. Useful for tags such as LINK and TR.
	 * <p>
	 * The <code>alttxt</code> parameter is related to the links' text specified
	 * by <code>title</code> elements. It assumes one of the following value:
	 * <ul>
	 * <li>{@link #ALTTXT_IGNORE}: alternate text are ignored
	 * <li>{@link #ALTTXT_INCLUDEONEMPTY}: alternate text are included only if
	 * the link anchor is empty
	 * <li>{@link #ALTTXT_INLCUDE}: alternate text is always retrieved and
	 * inserted before the link anchor
	 * </ul>
	 * <p>
	 * HTML <code>alt</code> elements (mandatory for IMG and AREA elements and
	 * optional for the INPUT and APPLET according to HTML4.01) are always
	 * included unless {@link #ALTTXT_IGNORE} is specified.
	 * 
	 * @see HypertextUnit#getText().
	 */
	public String getEnclosedText(int alttxt) {
		StringBuffer buf = new StringBuffer();
		if (text != null) {
			buf.append(text);
		}
		boolean b = type.equalsIgnoreCase("LINK")
				|| type.equalsIgnoreCase("IMG")
				|| type.equalsIgnoreCase("INPUT")
				|| type.equalsIgnoreCase("APPLET")
				|| type.equalsIgnoreCase("AREA");
		// collect of the text from the units included in this unit
		// but do not consider other descendants
		Iterator iter = content.iterator();
		while (iter.hasNext()) {
			HypertextUnit item = (HypertextUnit) iter.next();
			if (item.text == null)
				continue;
			if (b && item.type.equalsIgnoreCase("TEXT")) {
				if ((alttxt == HypertextUnit.ALTTXT_IGNORE)
						|| (text != null && alttxt == HypertextUnit.ALTTXT_INCLUDEONEMPTY))
					continue;
			}
			buf.append(" " + item.text);
		}
		return buf.toString().trim();
	}

	/**
	 * Returns an array of String objects corresponding to the text of the unit.
	 * <p>
	 * Each element of the array is the text of one descendant. If the unit
	 * contains some text, it is stored in the first element of the array.
	 * 
	 * @see HypertextUnit#enclosedText().
	 */
	// public String[] getEnclosedTextAsArray() {
	// ArrayList texts = new ArrayList();
	// if (text != null) {
	// texts.add(text);
	// }
	// // collect of the text from the units included in this unit
	// // but do not consider descendants
	// Iterator iter = content.iterator();
	// while (iter.hasNext()) {
	// HypertextUnit item = (HypertextUnit) iter.next();
	// if (item.text != null) {
	// texts.add(item.text);
	// }
	// }
	// return (String[]) texts.toArray(new String[] {});
	// }
	/**
	 * Collects the text from the current unit and all its descendants.
	 * <p>
	 * The <code>alttxt</code> parameter is related to the links' text specified
	 * by <code>title</code> elements. It assumes one of the following value:
	 * <ul>
	 * <li>{@link #ALTTXT_IGNORE}: alternate text are ignored
	 * <li>{@link #ALTTXT_INCLUDEONEMPTY}: alternate text are included only if
	 * the link anchor is empty
	 * <li>{@link #ALTTXT_INCLUDE}: alternate text is always retrieved and
	 * inserted before the link anchor
	 * </ul>
	 * <p>
	 * HTML <code>alt</code> elements (mandatory for IMG and AREA elements and
	 * optional for the INPUT and APPLET according to HTML4.01) are always
	 * included unless {@link #ALTTXT_IGNORE} is specified.
	 * 
	 * @see HypertextUnit#text().
	 */
	public String getNestedText(int alttxt) {
		StringBuffer buf = new StringBuffer();
		if (text != null) {
			buf.append(text);
		}
		boolean b = type.equalsIgnoreCase("LINK")
				|| type.equalsIgnoreCase("IMG")
				|| type.equalsIgnoreCase("INPUT")
				|| type.equalsIgnoreCase("APPLET")
				|| type.equalsIgnoreCase("AREA");
		Iterator iter = content.iterator();
		while (iter.hasNext()) {
			HypertextUnit item = (HypertextUnit) iter.next();
			if (b && item.type.equalsIgnoreCase("TEXT")) {
				if ((alttxt == HypertextUnit.ALTTXT_IGNORE)
						|| (text != null && alttxt == HypertextUnit.ALTTXT_INCLUDEONEMPTY))
					continue;
			}
			buf.append(" " + item.getNestedText(alttxt));
		}
		return buf.toString().trim();
	}

	/**
	 * Collects the text from the current unit and all its descendants.
	 * 
	 * @see HypertextUnit#text().
	 * @deprecated As for version 1.1, replaced by {@link #getNestedText(int)}.
	 */
	public String getNestedText() {
		return getNestedText(HypertextUnit.ALTTXT_INLCUDE);
	}

	/**
	 * Returns true if the unit <code>u</code> is a direct descendant of the
	 * node.
	 */
	public boolean isEnclosed(HypertextUnit u) {
		return content.contains(u);
	}

	/**
	 * Returns true if the unit <code>u</code> is one of the descendants of the
	 * node. Searching is performed with a recursive visit of the tree.
	 */
	public boolean isRecursivelyEnclosed(HypertextUnit u) {
		return recursiveIsEnclosed(this, u);
	}

	protected static boolean recursiveIsEnclosed(HypertextUnit currNode,
			HypertextUnit u) {
		if (currNode.content.contains(u)) {
			return true;
		}
		Iterator iter = currNode.content.iterator();
		while (iter.hasNext()) {
			if (recursiveIsEnclosed((HypertextUnit) iter.next(), u))
				return true;
		}
		return false;
	}

	/**
	 * A List that contains all children of this unit. If the goal is to
	 * recursively collect all the nodes below the current one see
	 * {@link #visit(int)}. To modify the tree structure, just add/remove nodes
	 * in the returned list. Beware that some information (e.g., the IDs) are
	 * assigned once the tree is built therefore they must be derived again if
	 * the structure is modified ({@link ClusterHypertextUnit#assignDepths()}.
	 */
	public List getChildUnits() {
		return content;
	}

	/**
	 * Overrides the childs of the node with the given not null list.
	 * 
	 * @param childs
	 */
	public void setChildUnits(List childs) {
		content = childs;
	}

	/**
	 * The parent of the unit, null if the unit is root.
	 */
	public HypertextUnit getParent() {
		return parent;
	}

	/**
	 * The parent of the unit, null if the unit is root.
	 */
	public HypertextUnit getRoot() {
		HypertextUnit prev = null, u = this;
		while (u != null) {
			prev = u;
			u = u.parent;
		}
		return prev != null ? prev : u;
	}

	/**
	 * The type of the nome. It can correspond to the name of the tag that
	 * generates the unit, or one of the following strings:
	 * <code>ROOT, TEXT, IMG, LINK</code>
	 */
	public String getType() {
		return type;
	}

	/**
	 * Returns the node depth.
	 * <p>
	 * The root node has depth 0.
	 */
	public int getDepth() {
		HypertextUnit p = parent;
		int d = 0;
		while (p != null) {
			p = parent.parent;
			d++;
		}
		return d;
	}

	/**
	 * The unit immediately following this unit.
	 */
	public HypertextUnit getNextSibling() {
		if (parent != null) {
			int idx = parent.content.indexOf(this) + 1;
			if (idx < parent.content.size())
				return (HypertextUnit) parent.content.get(idx);
		}
		return null;
	}

	/**
	 * The unit immediately preceding this unit.
	 */
	public HypertextUnit getPreviousSibling() {
		if (parent != null) {
			int idx = parent.content.indexOf(this) - 1;
			if (idx > 0)
				return (HypertextUnit) parent.content.get(idx);
		}
		return null;
	}

	/**
	 * The first child of this unit.
	 */
	public HypertextUnit getFirstChild() {
		if (content.size() > 0)
			return (HypertextUnit) content.get(0);
		return null;
	}

	/**
	 * The last child of this unit.
	 */
	public HypertextUnit getLastChild() {
		if (content.size() > 0)
			return (HypertextUnit) content.get(content.size() - 1);
		return null;
	}

	/**
	 * Visit the tree and call
	 * {@link HtmlContentAnalyzer#analyze(HypertextUnit, String)} for each node
	 * in a depth-first policy.
	 */
	public void analyzeText(HtmlContentAnalyzer analyzer) {
		text = analyzer.analyze(this, text);
		recursiveAnalyzeText(this, analyzer);
	}

	protected void recursiveAnalyzeText(HypertextUnit unit,
			HtmlContentAnalyzer a) {
		HypertextUnit u;
		List children = unit.getChildUnits();
		Iterator iter = children.iterator();
		while (iter.hasNext()) {
			u = (HypertextUnit) iter.next();
			u.text = a.analyze(u, u.text);
			recursiveAnalyzeText(u, a);
		}
	}

	/**
	 * Visits the current node and its sub-trees according to the parameter:
	 * <ol>
	 * <li>BREADTH_FIRST_TRAVERSAL
	 * <li>DEPTH_FIRST_TRAVERSAL (first the current node than its descendants,
	 * as in preorder)
	 * </ol>
	 * Note:
	 */
	public List visit(int traverseType) {
		List units = new ArrayList();
		units.add(this);
		switch (traverseType) {
		case HypertextUnit.DEPTH_FIRST_TRAVERSAL:
			units = depthVisit(this, units);
			break;
		case HypertextUnit.BREADTH_FIRST_TRAVERSAL:
			units = breadthVisit(this, units, new ArrayList(), 0);
			break;
		}
		return units;
	}

	protected List depthVisit(HypertextUnit unit, List units) {
		HypertextUnit u;
		List children = unit.getChildUnits();
		Iterator iter = children.iterator();
		while (iter.hasNext()) {
			u = (HypertextUnit) iter.next();
			units.add(u);
			depthVisit(u, units);
		}
		return units;
	}

	protected List breadthVisit(HypertextUnit unit, List units,
			List levelUnits, int level) {
		HypertextUnit u;
		List l = null;
		List children = unit.getChildUnits();
		if (levelUnits.size() > level)
			l = (List) levelUnits.get(level);
		if (l == null)
			l = new ArrayList();
		Iterator iter = children.iterator();
		while (iter.hasNext()) {
			u = (HypertextUnit) iter.next();
			l.add(u);
		}
		if (levelUnits.size() > level)
			levelUnits.set(level, l);
		else
			levelUnits.add(l);
		iter = children.iterator();
		while (iter.hasNext())
			breadthVisit((HypertextUnit) iter.next(), units, levelUnits,
					level + 1);
		if (level == 0) { // before quitting
			iter = levelUnits.iterator();
			while (iter.hasNext()) {
				units.addAll((List) iter.next());
			}
		}
		return units;

	}

	/**
	 * Looks for a unit given an Id through the descendants of the node.
	 * <p>
	 * Note: the id strings of the tree must be consistent before calling this method.
	 */
	public HypertextUnit retrieveUnit(String id) {
		HypertextUnit unit = this;
		try {
			// ignore root
			String newid = id.substring(2);
			int idx;
			while ((idx = newid.indexOf(':')) != -1) {
				String n = newid.substring(0, idx);
				newid = newid.substring(idx + 1);
				unit = (HypertextUnit) unit.content.get(Integer.parseInt(n));
			}
			unit = (HypertextUnit) unit.content.get(Integer.parseInt(newid));
		} catch (Exception ex) {
			return null;
		}
		return unit;
	}

	/**
	 * Returns the leafs of the sub-tree that starts from the node.
	 */
	public List getLeafs() {
		List l = recursiveLeafs(this, new ArrayList());
		return l;
		// return (HypertextUnit[]) l.toArray(new HypertextUnit[] {});
	}

	protected static List recursiveLeafs(HypertextUnit unit, List list) {
		if (unit.content.size() == 0) {
			list.add(unit);
			return list;
		}
		Iterator iter = unit.content.iterator();
		while (iter.hasNext()) {
			list = recursiveLeafs((HypertextUnit) iter.next(), list);
		}
		return list;
	}

	/**
	 * True if the unit is a leaf, that is, it has no children.
	 * <p>
	 * Corresponds to getContent().size()==0.
	 */
	public boolean isLeaf() {
		return (content.size() == 0);
	}

	/**
	 * Returns the first unit in the descendants of the node that correspond to
	 * the a given type.
	 */
	public HypertextUnit retrieveUnitByType(String type) {
		return recursiveRetrieveUnitByType(this, type);
	}

	protected static HypertextUnit recursiveRetrieveUnitByType(
			HypertextUnit unit, String type) {
		if (unit.type.equals(type)) {
			return unit;
		}
		Iterator iter = unit.content.iterator();
		while (iter.hasNext()) {
			HypertextUnit u = recursiveRetrieveUnitByType(
					(HypertextUnit) iter.next(), type);
			if (u != null)
				return u;
		}
		return null;
	}

	/**
	 * Returns the number of nodes in the sub-tree (1 if the node has not
	 * descendent).
	 */
	public int size() {
		return recursiveSize(this);
	}

	protected static int recursiveSize(HypertextUnit unit) {
		int n = 1;
		Iterator iter = unit.content.iterator();
		while (iter.hasNext()) {
			n += recursiveSize((HypertextUnit) iter.next());
		}
		return n;
	}

	/**
	 * Retrieve the units with the given url in the descendants of the node. If
	 * the url is malformed, returns an empty list.
	 * 
	 * The url matching is based on {@link URL#equals(Object)}.
	 */
	public List retrieveUnitsByLink(String url) {
		List l = null;
		try {
			l = recursiveRetrieveUnitsByLink(this, new URL(url),
					new ArrayList());
		} catch (MalformedURLException ex) {
			return new ArrayList();
		}
		return l;
	}

	protected static List recursiveRetrieveUnitsByLink(HypertextUnit unit,
			URL url1, List list) {
		try {
			URL url2 = new URL(unit.url);
			if ("LINK".equals(unit.type) && url1.equals(url2)) {
				list.add(unit);
			}
		} catch (MalformedURLException ex) {
		}
		Iterator iter = unit.content.iterator();
		while (iter.hasNext()) {
			list = recursiveRetrieveUnitsByLink((HypertextUnit) iter.next(),
					url1, list);
		}
		return list;
	}

	/**
	 * Analyzes the text in each tree node by means of a given lexical analyzer.
	 * <p>
	 * The text becomes null if the result of the lexical analysis is empty.
	 */
	public void textAnalysis(Analyzer analyzer) throws Exception {
		recursiveTextAnalysis(this, analyzer);
	}

	protected void recursiveTextAnalysis(HypertextUnit unit, Analyzer analyzer)
			throws Exception {
		if (unit.text != null) {
			unit.text = analyzeText(unit.text, analyzer);
		}
		Iterator iter = unit.content.iterator();
		while (iter.hasNext()) {
			HypertextUnit u = (HypertextUnit) iter.next();
			recursiveTextAnalysis(u, analyzer);
		}
	}

	/**
	 * Apply the given visitor to the current unit. The tree is visited in
	 * pre-order traversal.
	 */
	public void visitNodes(Visitor v) {
		v.beginParsing();
		recursiveVisitNodes(this, v);
		v.finishedParsing();
	}

	protected void recursiveVisitNodes(HypertextUnit unit, Visitor v) {
		v.visitTag(unit);
		if (!v.shouldRecurseChildren())
			return;
		Iterator iter = unit.content.iterator();
		while (iter.hasNext()) {
			HypertextUnit u = (HypertextUnit) iter.next();
			recursiveVisitNodes(u, v);
		}
	}

	/*
	 * Extracts terms from a string by means of a given lexical analyzer. <p> It
	 * returns null if the input text is null, or if it becomes empty after the
	 * lexical analysis.
	 */
	protected String analyzeText(String text, Analyzer analyzer)
			throws Exception {
		if (text == null)
			return null;

		StringBuffer newBuf = new StringBuffer();
		TokenStream tokenStream;
		String token;
		tokenStream = analyzer.tokenStream(new StringReader(text));

		token = tokenStream.next();
		while (token != null) {
			newBuf.append(token + " ");
			token = tokenStream.next();
		}
		text = newBuf.toString().trim();
		return text.length() > 0 ? text : null;
	}

	/**
	 * Represents the node and its immediate descendants as a readable string.
	 */
	public String toString() {
		StringBuffer buf = new StringBuffer();
		buf.append("<" + nodeContentToString() + " content:[");
		Iterator iter = content.iterator();
		while (iter.hasNext()) {
			HypertextUnit unit = (HypertextUnit) iter.next();
			buf.append(unit.toString());
		}
		buf.append("]>");
		return buf.toString();
	}

	/**
	 * Builds a simple html representations of the sub-tree.
	 */
	public String toHTML() {
		return recursiveToHTML(this);
	}

	protected static String recursiveToHTML(HypertextUnit u) {
		StringBuffer buf = new StringBuffer();
		if (u.type.equalsIgnoreCase("ROOT")) {
		} else if (u.type.equalsIgnoreCase("TITLE")) {
			buf.append("<title>" + u.text + "</title>");
		} else if (u.type.equalsIgnoreCase("TEXT")) {
			buf.append(u.text);
		} else if (u.type.equalsIgnoreCase("IMG")) {
			buf.append("<img src=\"" + u.url + "\"");
			if (u.text != null) {
				buf.append(" alt=\"" + u.text + "\"");
			}
			buf.append(">");
		} else if (u.type.equalsIgnoreCase("LINK")) {
			buf.append("<a href=\"" + u.url + "\"");
			if (u.text != null) {
				buf.append(" title=\"" + u.text + "\"");
			}
			buf.append(">");
		} else {
			buf.append("<" + u.type + ">");
		}

		Iterator iter = u.content.iterator();
		while (iter.hasNext()) {
			HypertextUnit unit = (HypertextUnit) iter.next();
			buf.append(unit.recursiveToHTML(unit));
		}

		if (u.type.equalsIgnoreCase("ROOT")) {
		} else if (u.type.equalsIgnoreCase("TITLE")) {
		} else if (u.type.equalsIgnoreCase("TEXT")) {
		} else if (u.type.equalsIgnoreCase("IMG")) {
		} else if (u.type.equalsIgnoreCase("LINK")) {
			buf.append("</a>");
		} else {
			buf.append("</" + u.type + ">");
		}

		return buf.toString();
	}

	/**
	 * Builds a simple html representations of the sub-tree where the html tags
	 * are included as visual comments.
	 */
	public String toCommentedHTML() {
		return recursiveToCommentedHTML(this, false);
	}

	protected static String recursiveToCommentedHTML(HypertextUnit u,
			boolean comment) {
		StringBuffer buf = new StringBuffer();
		if (u.type.equalsIgnoreCase("ROOT")) {
			// ignore root
		} else if (u.type.equalsIgnoreCase("TITLE")) {
			buf.append("<title>" + u.text + "</title>");
		} else if (u.type.equalsIgnoreCase("TEXT")) {
			buf.append(u.text);
		} else if (u.type.equalsIgnoreCase("IMG")) {
			buf.append("<font color=\"green\">&lt;img");
			if (u.text != null) {
				buf.append(" alt=\"" + u.text + "\"");
			}
			buf.append("&gt;</font>");
			buf.append("<img src=\"" + u.url + "\"");
			if (u.text != null) {
				buf.append(" alt=\"" + u.text + "\"");
			}
			buf.append(">");
		} else if (u.type.equalsIgnoreCase("LINK")) {
			buf.append("<font color=\"green\">&lt;a href");
			if (u.text != null) {
				buf.append(" title=\"" + u.text + "\"");
			}
			buf.append("&gt;</font>");
			buf.append("<a href=\"" + u.url + "\"");
			if (u.text != null) {
				buf.append(" title=\"" + u.text + "\"");
			}
			buf.append(">");
		} else {
			if (u.type.equalsIgnoreCase("BODY")) {
				comment = true;
			}
			if (comment) {
				buf.append("<font color=\"green\">&lt;" + u.type
						+ "&gt;</font><" + u.type + ">");
			} else {
				buf.append("<" + u.type + ">");
			}
		}

		Iterator iter = u.content.iterator();
		while (iter.hasNext()) {
			HypertextUnit unit = (HypertextUnit) iter.next();
			buf.append(unit.recursiveToCommentedHTML(unit, comment));
		}

		if (u.type.equalsIgnoreCase("ROOT")) {
		} else if (u.type.equalsIgnoreCase("TITLE")) {
		} else if (u.type.equalsIgnoreCase("TEXT")) {
		} else if (u.type.equalsIgnoreCase("IMG")) {
		} else if (u.type.equalsIgnoreCase("LINK")) {
			buf.append("</a><font color=\"green\">&lt;/a&gt;</font>");
		} else {
			if (comment) {
				buf.append("</" + u.type + "><font color=\"green\">&lt;/"
						+ u.type + "&gt;</font>");

			} else {
				buf.append("</" + u.type + ">");

			}
		}

		return buf.toString();
	}

	/**
	 * Represents the sub-tree of the node as a readable string.
	 */
	public String toIndentString() {
		return recursiveToIndentString(0);
	}

	protected String recursiveToIndentString(int indent) {
		StringBuffer buf = new StringBuffer();
		buf.append("<" + nodeContentToString() + " content:[");
		Iterator iter = content.iterator();
		while (iter.hasNext()) {
			HypertextUnit unit = (HypertextUnit) iter.next();
			buf.append("\n");
			for (int i = 0; i < indent; i++)
				buf.append("\t");
			buf.append(unit.recursiveToIndentString(indent + 1));
		}
		buf.append("]>");
		return buf.toString();
	}

	/*
	 * Overrides if you want to show user-defined fields.
	 */
	protected String nodeContentToString() {
		StringBuffer attrs = new StringBuffer();
		for (int i = 0; i < this.attributes.length; i++) {
			attrs.append(this.attributes[i] + " ");
		}
		return new String("id:" + id + " type:" + type + " attributes:("
				+ attrs.toString().trim() + ") url:" + url + " text:'" + text
				+ "'");
	}

	private void writeObject(java.io.ObjectOutputStream out) throws IOException {
		// out.defaultWriteObject();
		out.writeObject(id);
		out.writeObject(url != null? url : "");
		out.writeObject(text != null? text : "");		
		// it looks like writing int rises java.io.OptionalDataException
		out.writeObject(type);
		out.writeObject(new Integer(offsetLeft));
		out.writeObject(new Integer(offsetTop));
		out.writeObject(new Integer(width));
		out.writeObject(new Integer(height));
		out.writeObject(new Integer(content.size()));
		Iterator iter = content.iterator();
		while (iter.hasNext()) {
			HypertextUnit unit = (HypertextUnit) iter.next();
			out.writeObject(unit);
		}
		// out.writeObject(content);
		out.writeObject(new Integer(attributes.length));
		for (int i = 0; i < attributes.length; i++) {
			out.writeObject(attributes[i]);
		}
		
//		System.out.println("Finished writing HypertextUnit writeObject: "
//				+ this.id);

	}

	private void readObject(java.io.ObjectInputStream in) throws IOException,
			ClassNotFoundException {
		// in.defaultReadObject();
		id = (String) in.readObject();
//		System.out.println("Start reading HypertextUnit readObject: "
//				+ this.id);
		url = (String) in.readObject();
		text = (String) in.readObject();
		type = (String) in.readObject();
		offsetLeft = ((Integer)in.readObject()).intValue();
		offsetTop = ((Integer)in.readObject()).intValue();
		width = ((Integer)in.readObject()).intValue();
		height = ((Integer)in.readObject()).intValue();
		int size = ((Integer)in.readObject()).intValue();
		content = new ArrayList();
		HypertextUnit u = null;
		while (size-- > 0) {
			// it should work also in subclassing
//			try {
//				u = (HypertextUnit) this.getClass().newInstance();
//			} catch (InstantiationException ex) {
//				// never raised
//			} catch (IllegalAccessException ex) {
//				// never raised
//			}
//			u.readObject(in);
			u = (HypertextUnit)in.readObject();
			u.parent = this;
			content.add(u);
		}
		size = ((Integer)in.readObject()).intValue();
		attributes = new HypertextUnitAttribute[size];
		for (int i = 0; i < size; i++) {
			attributes[i] = (HypertextUnitAttribute)in.readObject();
		}
		//rebind();
//		System.out.println("Finished reading HypertextUnit readObject: "
//				+ this.id);

		// // read descendants
		// parent = null;
	}

	/**
	 * Rebuild parent-child relationship of the current node's childs after
	 * loading the object from a stream.
	 */
	protected void rebind() {
		Iterator iter = content.iterator();
		while (iter.hasNext()) {
			HypertextUnit unit = (HypertextUnit) iter.next();
			unit.parent = this;
		}
	}

	/**
	 * Performs a shallow clone of the content with exception of the List object
	 * that contains the children. The idea is that no one should modify the
	 * unit created by the html-unit builder. Of course, during the developing
	 * of libraries working on html structures, it is nice to have access and
	 * modify the unit fields. Unit deep-cloning is time expensive, so we clone
	 * just lists while their elements are in common between the original and
	 * the cloned objects. See {@link #deepClone()}.
	 * <p>
	 * Beware that references to the parent node is maintained (that is, the
	 * parent of the new node correspond with the parent of the original node).
	 */
	public Object clone() throws CloneNotSupportedException {
		HypertextUnit cloned = (HypertextUnit) super.clone();
		// copy descendant references
		cloned.content = new ArrayList();
		cloned.content.addAll(content);
		/*
		 * fields that contain primitive or immutable values (int, String,
		 * float, etc���) are correctly cloned by the java runtime
		 */
		return cloned;
	}

	/**
	 * Same as {@link #clone()} but it makes a recursive clone of all the
	 * children of the current node (till the leafs). The parent of the new node
	 * correspond with the parent of the original node.
	 */
	public Object deepClone() throws CloneNotSupportedException {
		HypertextUnit cloned = (HypertextUnit) super.clone();
		// copy descendant references
		cloned.content = new ArrayList();
		Iterator iter = content.iterator();
		while (iter.hasNext()) {
			HypertextUnit unit = (HypertextUnit) iter.next();
			cloned.content.add(unit.deepClone());
		}
		cloned.attributes = new HypertextUnitAttribute[this.attributes.length];
		for (int i = 0; i < this.attributes.length; i++) {
			cloned.attributes[i] = (HypertextUnitAttribute) this.attributes[i]
					.clone();
		}
		/*
		 * fields that contain primitive or immutable values (int, String,
		 * float, etc���) are correctly cloned by the java runtime
		 */
		return cloned;
	}

	/**
	 * Compare two HypertextUnit objects. The comparison is based on the
	 * <code>id</code> and the <code>type</code>.
	 * 
	 * Note: the id of the object must be set up.
	 * 
	 * @param unit
	 * @return
	 * @throws ClassCastException
	 */
	public int compareTo(Object unit) throws ClassCastException {
		if (!(unit instanceof HypertextUnit))
			throw new ClassCastException("A HypertextUnit object is expected.");
		HypertextUnit u = (HypertextUnit) unit;
		int cmp = id.compareTo(u.id);
		if (cmp == 0) {
			cmp = type.compareTo(u.type);
		}
		return cmp;
	}

	/* --- SETTERS --- */
	public void setHeight(int height) {
		this.height = height;
	}

	public void setId(String id) {
		this.id = id;
	}

	public void setOffsetLeft(int offsetLeft) {
		this.offsetLeft = offsetLeft;
	}

	public void setOffsetTop(int offsetTop) {
		this.offsetTop = offsetTop;
	}

	public void setParent(HypertextUnit parent) {
		this.parent = parent;
	}

	public void setText(String text) {
		this.text = text;
	}

	public void setType(String type) {
		this.type = type;
	}

	public void setUrl(String url) {
		this.url = url;
	}

	public void setWidth(int width) {
		this.width = width;
	}

	/**
	 * A not null array of tag attributes.
	 */
	public HypertextUnitAttribute[] getAttributes() {
		return this.attributes;
	}
	
	
	/**
	 * 
	 * @param id
	 * @return an empty String if the given id is the root
	 */
	public static String parentId(String id) {
		// ignore root
		String newid = "";
		int idx;
		if ((idx = id.lastIndexOf(':')) != -1) {
			newid = id.substring(0, idx);
		}
		return newid;		
	}
	

	public static void main(String[] args) throws Exception {
		// LogManager.getLogManager().reset();

		// Logger globalLogger =
		// Logger.getLogger(java.util.logging.Logger.GLOBAL_LOGGER_NAME);
		// globalLogger.setLevel(java.util.logging.Level.INFO);

		FileInputStream config = new FileInputStream("logging.properties");
		LogManager.getLogManager().readConfiguration(config);
		config.close();

		HypertextUnitBuilder builder = new HtmlParserHypertextUnitBuilder();
		HypertextUnitFactory factory = new DefaultHypertextUnitFactory();
		
		org.tabularium.net.downloader.Resource res = Downloader.retrieve("http://www.cnn.com/");
		String page = new String(res.getObject());
		
		HypertextUnit root = (HypertextUnit) builder.build("http://www.cnn.com", page, factory);
		System.out.println(root);
		 ObjectOutputStream outToClient = new ObjectOutputStream(
                 new FileOutputStream("obj.data"));
         outToClient.writeObject(root);
         outToClient.flush();         
         outToClient.close();
         
         ObjectInputStream inFromServer = new ObjectInputStream(
                new FileInputStream("obj.data"));
         root = (HypertextUnit)inFromServer.readObject();
         System.out.println(root);
		//sc.findCorrelations();
	}

}